{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import random\n",
    "import nltk \n",
    "from nltk.tokenize import word_tokenize\n",
    "import datetime\n",
    "import time\n",
    "import random\n",
    "import itertools\n",
    "import numpy as np\n",
    "import pickle\n",
    "from numpy.linalg import cholesky\n",
    "from keras.utils.np_utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def newsample(nnn,ratio):\n",
    "    if ratio >len(nnn):\n",
    "        return random.sample(nnn*(ratio//len(nnn)+1),ratio)\n",
    "    else:\n",
    "        return random.sample(nnn,ratio)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_user_file(file='ClickData4.tsv',npratio=4):\n",
    "    userid_dict={}\n",
    "    with open(file) as f:\n",
    "        userdata=f.readlines()\n",
    "    for user in userdata:\n",
    "        line=user.strip().split('\\t')\n",
    "        userid=line[0]\n",
    "        if userid not in userid_dict:\n",
    "            userid_dict[userid]=len(userid_dict)\n",
    "    \n",
    "    all_train_id=[]\n",
    "    all_train_pn=[]    \n",
    "    all_label=[]\n",
    "    \n",
    "    all_test_id=[]\n",
    "    all_test_pn=[]    \n",
    "    all_test_label=[]\n",
    "    all_test_index=[]\n",
    "    \n",
    "    all_user_pos=[]\n",
    "    all_test_user_pos=[]\n",
    "    \n",
    "    for user in userdata:\n",
    "        line=user.strip().split('\\t')\n",
    "        userid=line[0]\n",
    "        if len(line)==4:\n",
    "            \n",
    "            impre=[x.split('#TAB#') for x in line[2].split('#N#')]\n",
    "        if len(line)==3:\n",
    "            impre=[x.split('#TAB#') for x in line[2].split('#N#')]\n",
    "    \n",
    "        trainpos=[x[0].split() for x in impre]\n",
    "        trainneg=[x[1].split() for x in impre]\n",
    "         \n",
    "        poslist=list(itertools.chain(*(trainpos)))\n",
    "        neglist=list(itertools.chain(*(trainneg)))\n",
    "    \n",
    "        \n",
    "        if len(line)==4:\n",
    "            testimpre=[x.split('#TAB#') for x in line[3].split('#N#')]\n",
    "            testpos=[x[0].split() for x in testimpre]\n",
    "            testneg=[x[1].split() for x in testimpre]\n",
    "            \n",
    "            \n",
    "            for i in range(len(testpos)):\n",
    "                sess_index=[]\n",
    "                sess_index.append(len(all_test_pn))\n",
    "                posset=list(set(poslist))\n",
    "                allpos=[int(p) for p in random.sample(posset,min(50,len(posset)))[:50]]\n",
    "                allpos+=[0]*(50-len(allpos))\n",
    "        \n",
    "                \n",
    "                for j in testpos[i]:\n",
    "                    all_test_pn.append(int(j))\n",
    "                    all_test_label.append(1)\n",
    "                    all_test_id.append(userid_dict[userid])\n",
    "                    all_test_user_pos.append(allpos)\n",
    "                    \n",
    "                for j in testneg[i]:\n",
    "                    all_test_pn.append(int(j))\n",
    "                    all_test_label.append(0)\n",
    "                    all_test_id.append(userid_dict[userid])\n",
    "                    all_test_user_pos.append(allpos)\n",
    "                sess_index.append(len(all_test_pn))\n",
    "                all_test_index.append(sess_index)\n",
    "                \n",
    "    \n",
    "    \n",
    "                \n",
    "        for impre_id in range(len(trainpos)):\n",
    "            for pos_sample in trainpos[impre_id]:\n",
    "    \n",
    "                pos_neg_sample=newsample(trainneg[impre_id],npratio)\n",
    "                pos_neg_sample.append(pos_sample)\n",
    "                temp_label=[0]*npratio+[1]\n",
    "                temp_id=list(range(npratio+1))\n",
    "                random.shuffle(temp_id)\n",
    "    \n",
    "                \n",
    "                shuffle_sample=[]\n",
    "                shuffle_label=[]\n",
    "                for id in temp_id:\n",
    "                    shuffle_sample.append(int(pos_neg_sample[id]))\n",
    "                    shuffle_label.append(temp_label[id])\n",
    "                \n",
    "                posset=list(set(poslist)-set([pos_sample]))\n",
    "                allpos=[int(p) for p in random.sample(posset,min(50,len(posset)))[:50]]\n",
    "                allpos+=[0]*(50-len(allpos))\n",
    "                all_train_pn.append(shuffle_sample)\n",
    "                all_label.append(shuffle_label)\n",
    "                all_train_id.append(userid_dict[userid])\n",
    "                all_user_pos.append(allpos)\n",
    "            \n",
    "    all_train_pn=np.array(all_train_pn,dtype='int32')\n",
    "    all_label=np.array(all_label,dtype='int32')\n",
    "    all_train_id=np.array(all_train_id,dtype='int32')\n",
    "    all_test_pn=np.array(all_test_pn,dtype='int32')\n",
    "    all_test_label=np.array(all_test_label,dtype='int32')\n",
    "    all_test_id=np.array(all_test_id,dtype='int32')\n",
    "    all_user_pos=np.array(all_user_pos,dtype='int32')\n",
    "    all_test_user_pos=np.array(all_test_user_pos,dtype='int32')\n",
    "    return userid_dict,all_train_pn,all_label,all_train_id,all_test_pn,all_test_label,all_test_id,all_user_pos,all_test_user_pos,all_test_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_news_file(file='DocMeta3.tsv'):\n",
    "    with open(file) as f:\n",
    "        newsdata=f.readlines()\n",
    "    \n",
    "    news={}\n",
    "    category={'None':0}\n",
    "    subcategory={'None':0}\n",
    "\n",
    "    for newsline in newsdata:\n",
    "        line=newsline.strip().split('\\t')\n",
    "        news[line[1]]=[line[2],line[3],word_tokenize(line[6].lower()),word_tokenize(line[7].lower())]\n",
    "        if line[2] not in category:\n",
    "            category[line[2]]=len(category)\n",
    "        if line[3] not in subcategory:\n",
    "            subcategory[line[3]]=len(subcategory)\n",
    "            \n",
    "    word_dict_raw={'PADDING':[0,999999]}\n",
    "    \n",
    "    for docid in news:\n",
    "        for word in news[docid][2]:\n",
    "            if word in word_dict_raw:\n",
    "                word_dict_raw[word][1]+=1\n",
    "            else:\n",
    "                word_dict_raw[word]=[len(word_dict_raw),1]\n",
    "        for word in news[docid][3]:\n",
    "            if word in word_dict_raw:\n",
    "                word_dict_raw[word][1]+=1\n",
    "            else:\n",
    "                word_dict_raw[word]=[len(word_dict_raw),1]\n",
    "    word_dict={}\n",
    "    for i in word_dict_raw:\n",
    "        if word_dict_raw[i][1]>=3:\n",
    "            word_dict[i]=[len(word_dict),word_dict_raw[i][1]]\n",
    "    print(len(word_dict),len(word_dict_raw))\n",
    "    \n",
    "    news_words=[[0]*30]\n",
    "    news_index={'0':0}\n",
    "    for newsid in news:\n",
    "        word_id=[]\n",
    "        news_index[newsid]=len(news_index)\n",
    "        for word in news[newsid][2]:\n",
    "            if word in word_dict:\n",
    "                word_id.append(word_dict[word][0])\n",
    "        word_id=word_id[:30]\n",
    "        news_words.append(word_id+[0]*(30-len(word_id)))\n",
    "    news_words=np.array(news_words,dtype='int32') \n",
    "    \n",
    "    news_body=[[0]*300]\n",
    "    for newsid in news:\n",
    "        word_id=[]\n",
    "        for word in news[newsid][3]:\n",
    "            if word in word_dict:\n",
    "                word_id.append(word_dict[word][0])\n",
    "        word_id=word_id[:300]\n",
    "        news_body.append(word_id+[0]*(300-len(word_id)))\n",
    "    news_body=np.array(news_body,dtype='int32')     \n",
    "    news_v=[[0]]\n",
    "    news_sv=[[0]]\n",
    "    for newsid in news:\n",
    "        news_v.append([category[news[newsid][0]]])\n",
    "    for newsid in news:\n",
    "        news_sv.append([subcategory[news[newsid][1]]])\n",
    "    news_v=np.array(news_v,dtype='int32') \n",
    "    news_sv=np.array(news_sv,dtype='int32')     \n",
    "    return word_dict,category,subcategory,news_words,news_body,news_v,news_sv,news_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_embedding(word_dict):\n",
    "    embedding_dict={}\n",
    "    cnt=0\n",
    "    with open('/data/wuch/glove.840B.300d.txt','rb')as f:\n",
    "        linenb=0\n",
    "        while True:\n",
    "            line=f.readline()\n",
    "            if len(line)==0:\n",
    "                break\n",
    "            line = line.split()\n",
    "            word=line[0].decode()\n",
    "            linenb+=1\n",
    "            if len(word) != 0:\n",
    "                vec=[float(x) for x in line[1:]]\n",
    "                if word in word_dict:\n",
    "                    embedding_dict[word]=vec\n",
    "                    if cnt%1000==0:\n",
    "                        print(cnt,linenb,word)\n",
    "                    cnt+=1\n",
    "\n",
    "    embedding_matrix=[0]*len(word_dict)\n",
    "    cand=[]\n",
    "    for i in embedding_dict:\n",
    "        embedding_matrix[word_dict[i][0]]=np.array(embedding_dict[i],dtype='float32')\n",
    "        cand.append(embedding_matrix[word_dict[i][0]])\n",
    "    cand=np.array(cand,dtype='float32')\n",
    "    mu=np.mean(cand, axis=0)\n",
    "    Sigma=np.cov(cand.T)\n",
    "    norm=np.random.multivariate_normal(mu, Sigma, 1)\n",
    "    for i in range(len(embedding_matrix)):\n",
    "        if type(embedding_matrix[i])==int:\n",
    "            embedding_matrix[i]=np.reshape(norm, 300)\n",
    "    embedding_matrix[0]=np.zeros(300,dtype='float32')\n",
    "    embedding_matrix=np.array(embedding_matrix,dtype='float32')\n",
    "    print(embedding_matrix.shape)\n",
    "    return embedding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "userid_dict,all_train_pn,all_label,all_train_id,all_test_pn,all_test_label,all_test_id,all_user_pos,all_test_user_pos,all_test_index=preprocess_user_file()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_dict,category,subcategory,news_words,news_body,news_v,news_sv,news_index=preprocess_news_file()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_mat=get_embedding(word_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def dcg_score(y_true, y_score, k=10):\n",
    "    order = np.argsort(y_score)[::-1]\n",
    "    y_true = np.take(y_true, order[:k])\n",
    "    gains = 2 ** y_true - 1\n",
    "    discounts = np.log2(np.arange(len(y_true)) + 2)\n",
    "    return np.sum(gains / discounts)\n",
    "\n",
    "\n",
    "def ndcg_score(y_true, y_score, k=10):\n",
    "    best = dcg_score(y_true, y_true, k)\n",
    "    actual = dcg_score(y_true, y_score, k)\n",
    "    return actual / best\n",
    "\n",
    "\n",
    "def mrr_score(y_true, y_score):\n",
    "    order = np.argsort(y_score)[::-1]\n",
    "    y_true = np.take(y_true, order)\n",
    "    rr_score = y_true / (np.arange(len(y_true)) + 1)\n",
    "    return np.sum(rr_score) / np.sum(y_true)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n",
    "import keras\n",
    "from keras.layers import *\n",
    "from keras.models import Model\n",
    "from keras import backend as K\n",
    "from keras.optimizers import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def generate_batch_data_train(all_train_pn,all_label,all_train_id,batch_size):\n",
    "    inputid = np.arange(len(all_label))\n",
    "    np.random.shuffle(inputid)\n",
    "    y=all_label\n",
    "    batches = [inputid[range(batch_size*i, min(len(y), batch_size*(i+1)))] for i in range(len(y)//batch_size+1)]\n",
    "\n",
    "    while (True):\n",
    "        for i in batches:\n",
    "\n",
    "            candidate = news_words[all_train_pn[i]]\n",
    "            candidate_split=[candidate[:,k,:] for k in range(candidate.shape[1])]\n",
    "            candidate_body = news_body[all_train_pn[i]]\n",
    "            candidate_body_split=[candidate_body[:,k,:] for k in range(candidate_body.shape[1])]\n",
    "            candidate_vertical = news_v[all_train_pn[i]]\n",
    "            candidate_vertical_split=[candidate_vertical[:,k,:] for k in range(candidate_vertical.shape[1])]\n",
    "            candidate_subvertical = news_sv[all_train_pn[i]]\n",
    "            candidate_subvertical_split=[candidate_subvertical[:,k,:] for k in range(candidate_subvertical.shape[1])]\n",
    "            \n",
    "            browsed_news=news_words[all_user_pos[i]]\n",
    "            browsed_news_split=[browsed_news[:,k,:] for k in range(browsed_news.shape[1])]\n",
    "            browsed_news_body=news_body[all_user_pos[i]]\n",
    "            browsed_news_body_split=[browsed_news_body[:,k,:] for k in range(browsed_news_body.shape[1])]\n",
    "            browsed_news_vertical=news_v[all_user_pos[i]]\n",
    "            browsed_news_vertical_split=[browsed_news_vertical[:,k,:] for k in range(browsed_news_vertical.shape[1])]\n",
    "            browsed_news_subvertical=news_sv[all_user_pos[i]]\n",
    "            browsed_news_subvertical_split=[browsed_news_subvertical[:,k,:] for k in range(browsed_news_subvertical.shape[1])]\n",
    "            \n",
    "        \n",
    "            label=all_label[i]\n",
    "\n",
    "            yield (candidate_split+browsed_news_split+candidate_body_split+browsed_news_body_split\n",
    "                   +candidate_vertical_split+browsed_news_vertical_split +candidate_subvertical_split+browsed_news_subvertical_split, [label])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_batch_data_test(all_test_pn,all_label,all_test_id,batch_size):\n",
    "    inputid = np.arange(len(all_label))\n",
    "    y=all_label\n",
    "    batches = [inputid[range(batch_size*i, min(len(y), batch_size*(i+1)))] for i in range(len(y)//batch_size+1)]\n",
    "\n",
    "    while (True):\n",
    "        for i in batches:\n",
    "            candidate = news_words[all_test_pn[i]]\n",
    "            candidate_body = news_body[all_test_pn[i]]\n",
    "            candidate_vertical = news_v[all_test_pn[i]]\n",
    "            candidate_subvertical = news_sv[all_test_pn[i]]\n",
    "\n",
    "            browsed_news=news_words[all_test_user_pos[i]]\n",
    "            browsed_news_split=[browsed_news[:,k,:] for k in range(browsed_news.shape[1])]\n",
    "            browsed_news_body=news_body[all_test_user_pos[i]]\n",
    "            browsed_news_body_split=[browsed_news_body[:,k,:] for k in range(browsed_news_body.shape[1])]\n",
    "            browsed_news_vertical=news_v[all_test_user_pos[i]]\n",
    "            browsed_news_vertical_split=[browsed_news_vertical[:,k,:] for k in range(browsed_news_vertical.shape[1])]\n",
    "            browsed_news_subvertical=news_sv[all_test_user_pos[i]]\n",
    "            browsed_news_subvertical_split=[browsed_news_subvertical[:,k,:] for k in range(browsed_news_subvertical.shape[1])]\n",
    "            \n",
    "        \n",
    "            label=all_label[i]\n",
    "            yield ([candidate]+ browsed_news_split+[candidate_body]+ browsed_news_body_split+[candidate_vertical]\n",
    "                   + browsed_news_vertical_split+[candidate_subvertical]+ browsed_news_subvertical_split, [yy])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "import keras\n",
    "import random\n",
    "results=[]\n",
    "keras.backend.clear_session()\n",
    "\n",
    "MAX_SENT_LENGTH=30\n",
    "MAX_SENTS=50\n",
    "npratio=4\n",
    "\n",
    "\n",
    "\n",
    "MAX_BODY_LENGTH=300\n",
    "title_input = Input(shape=(MAX_SENT_LENGTH,), dtype='int32')\n",
    "\n",
    "body_input = Input(shape=(MAX_BODY_LENGTH,), dtype='int32')\n",
    "embedding_layer = Embedding(len(word_dict), 300, weights=[embedding_mat],trainable=True)\n",
    "\n",
    "embedded_sequences_title = embedding_layer(title_input)\n",
    "embedded_sequences_title=Dropout(0.2)(embedded_sequences_title)\n",
    "\n",
    "embedded_sequences_body = embedding_layer(body_input)\n",
    "embedded_sequences_body=Dropout(0.2)(embedded_sequences_body)\n",
    "\n",
    "title_cnn = Convolution1D(nb_filter=400, filter_length=3,  padding='same', activation='relu', strides=1)(embedded_sequences_title)\n",
    "title_cnn=Dropout(0.2)(title_cnn)\n",
    "\n",
    "attention = Dense(200,activation='tanh')(title_cnn)\n",
    "attention = Flatten()(Dense(1)(attention))\n",
    "attention_weight = Activation('softmax')(attention)\n",
    "title_rep=keras.layers.Dot((1, 1))([title_cnn, attention_weight])\n",
    "\n",
    "body_cnn = Convolution1D(nb_filter=400, filter_length=3,  padding='same', activation='relu', strides=1)(embedded_sequences_body)\n",
    "body_cnn=Dropout(0.2)(body_cnn)\n",
    "\n",
    "attention_body = Dense(200,activation='tanh')(body_cnn)\n",
    "attention_body = Flatten()(Dense(1)(attention_body))\n",
    "attention_weight_body = Activation('softmax')(attention_body)\n",
    "body_rep=keras.layers.Dot((1, 1))([body_cnn, attention_weight_body])\n",
    "\n",
    "vinput=Input((1,), dtype='int32') \n",
    "svinput=Input((1,), dtype='int32') \n",
    "v_embedding_layer = Embedding(len(category)+1, 50,trainable=True)\n",
    "sv_embedding_layer = Embedding(len(subcategory)+1, 50,trainable=True)\n",
    "v_embedding=Dense(400,activation='relu')(Flatten()(v_embedding_layer(vinput)))\n",
    "sv_embedding=Dense(400,activation='relu')(Flatten()(sv_embedding_layer(svinput)))\n",
    "\n",
    "all_channel=[title_rep,body_rep,v_embedding,sv_embedding]\n",
    "    \n",
    "views=concatenate([Lambda(lambda x: K.expand_dims(x,axis=1))(channel) for channel in all_channel],axis=1)\n",
    "\n",
    "attentionv = Dense(200,activation='tanh')(views)\n",
    "\n",
    "attention_weightv =Lambda(lambda x:K.squeeze(x,axis=-1))(Dense(1)(attentionv))\n",
    "attention_weightv =Activation('softmax')(attention_weightv)\n",
    "\n",
    "newsrep=keras.layers.Dot((1, 1))([views, attention_weightv])\n",
    "\n",
    "newsEncoder = Model([title_input,body_input,vinput,svinput],newsrep)\n",
    "\n",
    "browsed_news_input = [keras.Input((MAX_SENT_LENGTH,), dtype='int32') for _ in range(MAX_SENTS)]\n",
    "browsed_body_input = [keras.Input((MAX_BODY_LENGTH,), dtype='int32') for _ in range(MAX_SENTS)]\n",
    "\n",
    "browsed_v_input = [keras.Input((1,), dtype='int32') for _ in range(MAX_SENTS)]\n",
    "browsed_sv_input = [keras.Input((1,), dtype='int32') for _ in range(MAX_SENTS)]\n",
    "\n",
    "browsednews = [newsEncoder([browsed_news_input[_],browsed_body_input[_],browsed_v_input[_],browsed_sv_input[_] ]) for _ in range(MAX_SENTS)]\n",
    "browsednewsrep =concatenate([Lambda(lambda x: K.expand_dims(x,axis=1))(news) for news in browsednews],axis=1)    \n",
    "\n",
    "attentionn = Dense(200,activation='tanh')(browsednewsrep)\n",
    "attentionn =Flatten()(Dense(1)(attentionn))\n",
    "attention_weightn = Activation('softmax')(attentionn)\n",
    "user_rep=keras.layers.Dot((1, 1))([browsednewsrep, attention_weightn])\n",
    "\n",
    "candidates_title = [keras.Input((MAX_SENT_LENGTH,), dtype='int32') for _ in range(1+npratio)]\n",
    "\n",
    "candidates_body = [keras.Input((MAX_BODY_LENGTH,), dtype='int32') for _ in range(1+npratio)]\n",
    "\n",
    "candidates_v = [keras.Input((1,), dtype='int32') for _ in range(1+npratio)]\n",
    "\n",
    "candidates_sv = [keras.Input((1,), dtype='int32') for _ in range(1+npratio)]\n",
    "candidate_vecs = [newsEncoder([candidates_title[_],candidates_body[_],candidates_v[_],candidates_sv[_]]) for _ in range(1+npratio)]\n",
    "\n",
    "logits = [keras.layers.dot([user_rep, candidate_vec], axes=-1) for candidate_vec in candidate_vecs]\n",
    "logits = keras.layers.Activation(keras.activations.softmax)(keras.layers.concatenate(logits))\n",
    "\n",
    "\n",
    "model = Model(candidates_title+browsed_news_input+candidates_body+browsed_body_input+\n",
    "              candidates_v+browsed_v_input+candidates_sv+browsed_sv_input, logits)\n",
    "\n",
    "\n",
    "candidate_one_title = keras.Input((MAX_SENT_LENGTH,))\n",
    "\n",
    "candidate_one_body = keras.Input((MAX_BODY_LENGTH,))\n",
    "\n",
    "candidate_one_v = keras.Input((1,))\n",
    "\n",
    "candidate_one_sv = keras.Input((1,))\n",
    "\n",
    "candidate_one_vec=newsEncoder([candidate_one_title,candidate_one_body,candidate_one_v,candidate_one_sv])\n",
    "\n",
    "score = keras.layers.Activation(keras.activations.sigmoid)(keras.layers.dot([user_rep, candidate_one_vec], axes=-1))\n",
    "model_test = keras.Model([candidate_one_title]+browsed_news_input+[candidate_one_body] +browsed_body_input\n",
    "                         +[candidate_one_v]+browsed_v_input+[candidate_one_sv]+browsed_sv_input, score)\n",
    "\n",
    "\n",
    "model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.001), metrics=['acc'])\n",
    "\n",
    "for ep in range(3):\n",
    "    traingen=generate_batch_data_train(all_train_pn,all_label,all_train_id, 30)\n",
    "    model.fit_generator(traingen, epochs=1,steps_per_epoch=len(all_train_id)//30)\n",
    "    testgen=generate_batch_data_test(all_test_pn,all_test_label,all_test_id, 30)\n",
    "    click_score = model_test.predict_generator(testgen, steps=len(all_test_id)//30,verbose=1)\n",
    "    from sklearn.metrics import roc_auc_score\n",
    "    all_auc=[]\n",
    "    all_mrr=[]\n",
    "    all_ndcg=[]\n",
    "    all_ndcg2=[]\n",
    "    for m in all_test_index:\n",
    "        if np.sum(all_test_label[m[0]:m[1]])!=0 and m[1]<len(click_score):\n",
    "            all_auc.append(roc_auc_score(all_test_label[m[0]:m[1]],click_score[m[0]:m[1],0]))\n",
    "            all_mrr.append(mrr_score(all_test_label[m[0]:m[1]],click_score[m[0]:m[1],0]))\n",
    "            all_ndcg.append(ndcg_score(all_test_label[m[0]:m[1]],click_score[m[0]:m[1],0],k=5))\n",
    "            all_ndcg2.append(ndcg_score(all_test_label[m[0]:m[1]],click_score[m[0]:m[1],0],k=10))\n",
    "    results.append([np.mean(all_auc),np.mean(all_mrr),np.mean(all_ndcg),np.mean(all_ndcg2)])\n",
    "    print(np.mean(all_auc),np.mean(all_mrr),np.mean(all_ndcg),np.mean(all_ndcg2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
